PART II a): German Credit Score Classification Model EXPLAINABILITY, BIAS & FAIRNESS (Age as protected variable).
By: Krishna J¶
Importing necessary libraries ¶
In [1]:
#!pip install --upgrade tensorflow==1.15.0
In [2]:
import pandas as pd
import numpy as np
import seaborn as sns
import matplotlib.pyplot as plt
from sklearn.model_selection import train_test_split
#from sklearn.ensemble import RandomForestClassifier
#from sklearn.linear_model import LogisticRegression
from sklearn.preprocessing import MinMaxScaler, StandardScaler
from sklearn.base import TransformerMixin
from sklearn.pipeline import Pipeline, FeatureUnion
from typing import List, Union, Dict
# Warnings will be used to silence various model warnings for tidier output
import warnings
warnings.filterwarnings('ignore')
%matplotlib inline
from IPython.core.interactiveshell import InteractiveShell
InteractiveShell.ast_node_interactivity = "all"
np.random.seed(0)
#!pip install fairlearn
#!pip install aif360
#!pip install shap
#!pip install eli5
#!pip install BlackBoxAuditing
Importing source dataset ¶
In [3]:
German_df = pd.read_csv('C:/Users/krish/Downloads/German-reduced_jp.csv')
print(German_df.shape)
print (German_df.columns)
In [4]:
German_df.head()
Out[4]:
In [5]:
#feature_list = ['Gender','Age','Marital_Status','NumMonths','Savings_<500','Savings_none','Dependents','Property_rent','Job_management/self-emp/officer/highly qualif emp','Debtors_guarantor','Purpose_CarNew', 'Purpose_furniture/equip','CreditHistory_none/paid','Purpose_CarUsed','CreditAmount','CreditStatus']
feature_list=['Gender', 'Age', 'Marital_Status', 'CurrentAcc_None',
'CurrentAcc_LT200', 'Savings_LT500', 'CreditHistory_none/paid',
'Debtors_co-applicant', 'Job_unskilled-resident', 'NumMonths',
'Telephone', 'Purpose_education', 'Purpose_furniture/equip',
'CreditAmount', 'Foreignworker', 'Debtors_guarantor', 'CreditStatus']
In [6]:
X = German_df.iloc[:, :-1]
y = German_df['CreditStatus']
X.head()
y.head()
Out[6]:
Out[6]:
from imblearn.over_sampling import ADASYN from collections import Counter
ada = ADASYN(random_state=40) print('Original dataset shape {}'.format(Counter(y))) X_res, y_res = ada.fit_resample(X,y) print('Resampled dataset shape {}'.format(Counter(y_res)))
German_df=X = pd.DataFrame(np.column_stack((X_res, y_res)))
In [7]:
German_df.head()
Out[7]:
German_df.columns=feature_list German_df.head()
Metrics to calculate model fairness necessary libraries ¶
In [8]:
from aif360.datasets import GermanDataset
from aif360.metrics import BinaryLabelDatasetMetric
def fair_metrics(fname, dataset, pred, pred_is_dataset=False):
filename = fname
if pred_is_dataset:
dataset_pred = pred
else:
dataset_pred = dataset.copy()
dataset_pred.labels = pred
cols = ['Accuracy', 'F1', 'DI','SPD', 'EOD', 'AOD', 'ERD', 'CNT', 'TI']
obj_fairness = [[1,1,1,0,0,0,0,1,0]]
fair_metrics = pd.DataFrame(data=obj_fairness, index=['objective'], columns=cols)
for attr in dataset_pred.protected_attribute_names:
idx = dataset_pred.protected_attribute_names.index(attr)
privileged_groups = [{attr:dataset_pred.privileged_protected_attributes[idx][0]}]
unprivileged_groups = [{attr:dataset_pred.unprivileged_protected_attributes[idx][0]}]
classified_metric = ClassificationMetric(dataset,
dataset_pred,
unprivileged_groups=unprivileged_groups,
privileged_groups=privileged_groups)
metric_pred = BinaryLabelDatasetMetric(dataset_pred,
unprivileged_groups=unprivileged_groups,
privileged_groups=privileged_groups)
distortion_metric = SampleDistortionMetric(dataset,
dataset_pred,
unprivileged_groups=unprivileged_groups,
privileged_groups=privileged_groups)
acc = classified_metric.accuracy()
f1_sc = 2 * (classified_metric.precision() * classified_metric.recall()) / (classified_metric.precision() + classified_metric.recall())
mt = [acc, f1_sc,
classified_metric.disparate_impact(),
classified_metric.mean_difference(),
classified_metric.equal_opportunity_difference(),
classified_metric.average_odds_difference(),
classified_metric.error_rate_difference(),
metric_pred.consistency(),
classified_metric.theil_index()
]
w_row = []
print('Computing fairness of the model.')
for i in mt:
#print("%.8f"%i)
w_row.append("%.8f"%i)
with open(filename, 'a') as csvfile:
csvwriter = csv.writer(csvfile)
csvwriter.writerow(w_row)
row = pd.DataFrame([mt],
columns = cols,
index = [attr]
)
fair_metrics = fair_metrics.append(row)
fair_metrics = fair_metrics.replace([-np.inf, np.inf], 2)
return fair_metrics
def get_fair_metrics_and_plot(fname, data, model, plot=False, model_aif=False):
pred = model.predict(data).labels if model_aif else model.predict(data.features)
fair = fair_metrics(fname, data, pred)
if plot:
pass
return fair
def get_model_performance(X_test, y_true, y_pred, probs):
accuracy = accuracy_score(y_true, y_pred)
matrix = confusion_matrix(y_true, y_pred)
f1 = f1_score(y_true, y_pred)
return accuracy, matrix, f1
def plot_model_performance(model, X_test, y_true):
y_pred = model.predict(X_test)
probs = model.predict_proba(X_test)
accuracy, matrix, f1 = get_model_performance(X_test, y_true, y_pred, probs)
Local file to load metric values¶
In [9]:
filename= 'C:/Users/krish/Downloads/main_pjt_final - Copy/filename_mainpjt_results_age_may6_jp.csv'
Converting data to aif compatible format¶
Since we are dealing with binary label dataset we are using aif360 class BiaryLabelDataset here with target label as CreditStatus and protected attributes as age,gender,marital status. Refer part 11 for more details on protected attributes and privileged classes.
In [10]:
# Fairness metrics
from aif360.metrics import BinaryLabelDatasetMetric
from aif360.explainers import MetricTextExplainer
from aif360.metrics import ClassificationMetric
# Get DF into IBM format
from aif360 import datasets
#converting to aif dataset
aif_dataset = datasets.BinaryLabelDataset(favorable_label = 1, unfavorable_label = 0, df=German_df,
label_names=["CreditStatus"],
protected_attribute_names=["Age"],
privileged_protected_attributes = [1])
In [11]:
#dataset_orig = GermanDataset(protected_attribute_names=['sex'],
# privileged_classes=[[1]],
# features_to_keep=['age', 'sex', 'employment', 'housing', 'savings', 'credit_amount', 'month', 'purpose'],
# custom_preprocessing=custom_preprocessing)
Splitting data to train and test sets¶
In [12]:
#privileged_groups = [{'Age':1},{' Gender': 1},{'Marital_Status':1}]
#unprivileged_groups = [{'Age':0},{'Gender': 0},{'Marital_Status':0}]
In [13]:
privileged_groups = [{'Age': 1}]
unprivileged_groups = [{'Age': 0}]
In [14]:
data_orig_train, data_orig_test = aif_dataset.split([0.8], shuffle=True)
X_train = data_orig_train.features
y_train = data_orig_train.labels.ravel()
X_test = data_orig_test.features
y_test = data_orig_test.labels.ravel()
In [15]:
X_train.shape
X_test.shape
Out[15]:
Out[15]:
In [16]:
data_orig_test.labels[:10].ravel()
Out[16]:
In [17]:
data_orig_train.labels[:10].ravel()
Out[17]:
Testing bias with respect to protected variable¶
In [18]:
metric_orig_train = BinaryLabelDatasetMetric(data_orig_train,
unprivileged_groups=unprivileged_groups,
privileged_groups=privileged_groups)
print("Difference in mean outcomes between unprivileged and privileged groups = %f" % metric_orig_train.mean_difference())
A non zero value indicates bias.
Building ML model ¶
Considering ensemble models for our study.
1. RANDOM FOREST CLASSIFIER MODEL¶
In [19]:
#Seting the Hyper Parameters
param_grid = {"max_depth": [3,5,7, 10,None],
"n_estimators":[3,5,10,25,50,150],
"max_features": [4,7,15,20]}
from sklearn.ensemble import RandomForestClassifier
from sklearn.model_selection import GridSearchCV
#Creating the classifier
rf_model = RandomForestClassifier(random_state=40)
grid_search = GridSearchCV(rf_model, param_grid=param_grid, cv=5, scoring='recall', verbose=0)
model_rf = grid_search
In [20]:
mdl_rf = model_rf.fit(data_orig_train.features, data_orig_train.labels.ravel())
In [21]:
from sklearn.metrics import confusion_matrix
conf_mat_rf = confusion_matrix(data_orig_test.labels.ravel(), model_rf.predict(data_orig_test.features))
conf_mat_rf
from sklearn.metrics import accuracy_score
print(accuracy_score(data_orig_test.labels.ravel(), model_rf.predict(data_orig_test.features)))
Out[21]:
In [22]:
unique, counts = np.unique(data_orig_test.labels.ravel(), return_counts=True)
dict(zip(unique, counts))
Out[22]:
1.a. Feature importance of model¶
In [23]:
importances = model_rf.best_estimator_.feature_importances_
indices = np.argsort(importances)
features = data_orig_train.feature_names
#https://stackoverflow.com/questions/48377296/get-feature-importance-from-gridsearchcv
In [24]:
importances
Out[24]:
In [25]:
importances[indices]
Out[25]:
In [26]:
features
Out[26]:
In [27]:
plt.figure(figsize=(20,30))
plt.title('Feature Importances')
plt.barh(range(len(indices)), importances[indices], color='b', align='center')
plt.yticks(range(len(indices)), [features[i] for i in indices])
plt.xlabel('Relative Importance')
plt.show()
Out[27]:
Out[27]:
Out[27]:
Out[27]:
Out[27]:
1.b. Model Explainability/interpretability¶
1.b.1 Using SHAP (SHapley Additive exPlanations)¶
In [28]:
import shap
Test data interpretation¶
In [29]:
rf_explainer = shap.KernelExplainer(model_rf.predict, data_orig_test.features)
rf_shap_values = rf_explainer.shap_values(data_orig_test.features,nsamples=50)
#https://towardsdatascience.com/explain-any-models-with-the-shap-values-use-the-kernelexplainer-79de9464897a
In [30]:
rf_shap_values
Out[30]:
In [31]:
rf_explainer.expected_value
Out[31]:
In [32]:
y_test_predict=model_rf.predict(data_orig_test.features)
y_test_predict[:12]
data_orig_test.labels[:12].ravel()
data_orig_test.features[:2,:]
Out[32]:
Out[32]:
Out[32]:
In [33]:
y_test_predict.mean()
Out[33]:
The explainer expected value is the average model predicted value on input data. Shapely helps to understand how individual features impact the output of each individual instance. The shapely values are model predicted values which may not coincide with actual y test values due to prediction error.
link=”logit” argument converts the logit values to probability
In [34]:
shap.initjs()
shap.force_plot(rf_explainer.expected_value,rf_shap_values[0],data_orig_test.features[0],data_orig_test.feature_names,link='logit')
#https://github.com/slundberg/shap
#https://github.com/slundberg/shap/issues/279
#https://github.com/slundberg/shap/issues/977
shap.initjs()
shap.force_plot(rf_explainer.expected_value,rf_shap_values[0],data_orig_test.features[0],data_orig_test.feature_names)
Out[34]:
Out[34]:
Features in blue pushes the base value towards lowest values and features in red moves base levels towards higher values.
Shapley values calculate the importance of a feature by comparing what a model predicts with and without the feature. However, since the order in which a model sees features can affect its predictions, this is done in every possible order, so that the features are fairly compared.
The SHAP plot shows features that contribute to pushing the output from the base value (average model output) to the actual predicted value.
In [35]:
shap.initjs()
shap.force_plot(rf_explainer.expected_value,rf_shap_values[1], data_orig_test.features[1],data_orig_test.feature_names,link='logit')
shap.initjs()
shap.force_plot(rf_explainer.expected_value,rf_shap_values[1], data_orig_test.features[1],data_orig_test.feature_names)
Out[35]:
Out[35]:
In [36]:
data_orig_test.feature_names
Out[36]:
In [37]:
shap.force_plot(rf_explainer.expected_value,
rf_shap_values, data_orig_test.features[:,:],feature_names = data_orig_test.feature_names)
Out[37]:
In [38]:
p = shap.summary_plot(rf_shap_values, data_orig_test.features, feature_names=data_orig_test.feature_names,plot_type="bar")
display(p)
Variables with higher impact are displayed at the credit history, credit amount,num of months.
In [39]:
shap.decision_plot(rf_explainer.expected_value, rf_shap_values,feature_names=data_orig_test.feature_names)
- The x-axis represents the model's output. In this case, the units are log odds.
- The plot is centered on the x-axis at explainer.expected_value.
- All SHAP values are relative to the model's expected value like a linear model's effects are relative to the intercept.
- The y-axis lists the model's features.
- By default, the features are ordered by descending importance. The importance is calculated over the observations plotted. This is usually different than the importance ordering for the entire dataset.
- In addition to feature importance ordering, the decision plot also supports hierarchical cluster feature ordering and user-defined feature ordering.
- Each observation's prediction is represented by a colored line. At the top of the plot, each line strikes the x-axis at its corresponding observation's predicted value. This value determines the color of the line on a spectrum.
- Moving from the bottom of the plot to the top, SHAP values for each feature are added to the model's base value. This shows how each feature contributes to the overall prediction.
- At the bottom of the plot, the observations converge at explainer.expected_value https://slundberg.github.io/shap/notebooks/plots/decision_plot.html
Like the force plot, the decision plot supports link='logit' to transform log odds to probabilities.
In [40]:
shap.decision_plot(rf_explainer.expected_value, rf_shap_values,feature_names=data_orig_test.feature_names,link='logit')
In [41]:
shap.plots._waterfall.waterfall_legacy(rf_explainer.expected_value, rf_shap_values[0],feature_names=data_orig_test.feature_names)
For first instace of input,out of all the displayed variables, current account is playing major role is pushing the target variable outcome towards predicting 1.
Interpretation of graph: https://shap.readthedocs.io/en/latest/example_notebooks/overviews/An%20introduction%20to%20explainable%20AI%20with%20Shapley%20values.html
f(x)- model output impacted by features; E(f(x))- expected output.
One the fundemental properties of Shapley values is that they always sum up to the difference between the game outcome when all players are present and the game outcome when no players are present. For machine learning models this means that SHAP values of all the input features will always sum up to the difference between baseline (expected) model output and the current model output for the prediction being explained.
Shapley values calculate the importance of a feature by comparing what a model predicts with and without the feature. However, since the order in which a model sees features can affect its predictions, this is done in every possible order, so that the features are fairly compared. https://medium.com/@gabrieltseng/interpreting-complex-models-with-shap-values-1c187db6ec83
In [42]:
shap.plots._waterfall.waterfall_legacy(rf_explainer.expected_value, rf_shap_values[1],feature_names=data_orig_test.feature_names)
For second instace of input,out of all the displayed variables, credit amount is playing major role is pushing the target variable outcome towards predicting 0.
1.b.2 Using ELI5¶
In [43]:
#!pip install eli5
import eli5
In [44]:
from eli5.sklearn import PermutationImportance
In [45]:
perm_rf = PermutationImportance(mdl_rf).fit(data_orig_test.features, data_orig_test.labels.ravel())
Feature Importance¶
In [46]:
perm_imp_1=eli5.show_weights(perm_rf,feature_names = data_orig_test.feature_names)
perm_imp_1
plt.show()
Out[46]:
- eli5 provides a way to compute feature importances for any black-box estimator by measuring how score decreases when a feature is not available; the method is also known as “permutation importance” or “Mean Decrease Accuracy (MDA)”.
The first number in each row shows how much model performance decreased with a random shuffling (in this case, using "accuracy" as the performance metric).
Like most things in data science, there is some randomness to the exact performance change from a shuffling a column. We measure the amount of randomness in our permutation importance calculation by repeating the process with multiple shuffles. The number after the ± measures how performance varied from one-reshuffling to the next.
You'll occasionally see negative values for permutation importances. In those cases, the predictions on the shuffled (or noisy) data happened to be more accurate than the real data. This happens when the feature didn't matter (should have had an importance close to 0), but random chance caused the predictions on shuffled data to be more accurate. This is more common with small datasets, like the one in this example, because there is more room for luck/chance.
1.c. Measuring fairness ¶
Of Baseline model ¶
In [47]:
import pandas as pd
import csv
import os
import numpy as np
import sys
from aif360.metrics import *
from sklearn.metrics import confusion_matrix, accuracy_score, f1_score, roc_curve, auc
plot_model_performance(mdl_rf, X_test, y_test)
In [48]:
fair_rf = get_fair_metrics_and_plot(filename, data_orig_test, mdl_rf)
fair_rf
Out[48]:
In [49]:
type(data_orig_train)
Out[49]:
PRE PROCESSING¶
In [50]:
### Reweighing
from aif360.algorithms.preprocessing import Reweighing
RW_rf = Reweighing(unprivileged_groups=unprivileged_groups, privileged_groups=privileged_groups)
data_transf_train_rf_rw = RW_rf.fit_transform(data_orig_train)
#train and save model
rf_transf_rw = model_rf.fit(data_transf_train_rf_rw.features,
data_transf_train_rf_rw.labels.ravel())
data_transf_test_rf_rw = RW_rf.transform(data_orig_test)
fair_rf_rw = get_fair_metrics_and_plot(filename, data_transf_test_rf_rw, rf_transf_rw, plot=False)
In [51]:
metric_transf_train = BinaryLabelDatasetMetric(data_transf_train_rf_rw,
unprivileged_groups=unprivileged_groups,
privileged_groups=privileged_groups)
print("Difference in mean outcomes between unprivileged and privileged groups = %f" % metric_transf_train.mean_difference())
In [52]:
fair_rf_rw
Out[52]:
In [53]:
from aif360.algorithms.preprocessing import DisparateImpactRemover
DIR_rf = DisparateImpactRemover()
data_transf_train_rf_dir = DIR_rf.fit_transform(data_orig_train)
# Train and save the model
rf_transf_dir = model_rf.fit(data_transf_train_rf_dir.features,data_transf_train_rf_dir.labels.ravel())
In [54]:
fair_dir_rf_dir = get_fair_metrics_and_plot(filename,data_orig_test, rf_transf_dir, plot=False)
fair_dir_rf_dir
Out[54]:
INPROCESSING¶
In [55]:
#!pip install --user --upgrade tensorflow==1.15.0
#2.2.0
#!pip uninstall tensorflow
In [56]:
#!pip install "tensorflow==1.15"
#!pip install --upgrade tensorflow-hub
In [57]:
#%tensorflow_version 1.15
import tensorflow as tf
#from tensorflow.compat.v1 import variable_scope
print('Using TensorFlow version', tf.__version__)
In [58]:
#sess = tf.compat.v1.Session()
#import tensorflow as tf
sess = tf.compat.v1.Session()
In [59]:
#import tensorflow as tf
#sess = tf.Session()
tf.compat.v1.reset_default_graph()
In [60]:
from aif360.algorithms.inprocessing.adversarial_debiasing import AdversarialDebiasing
#with tf.variable_scope('debiased_classifier',reuse=tf.AUTO_REUSE):
with tf.compat.v1.Session() as sess:
with tf.variable_scope('scope1',reuse=tf.AUTO_REUSE) as scope:
debiased_model_rf_ad = AdversarialDebiasing(privileged_groups = privileged_groups,
unprivileged_groups = unprivileged_groups,
scope_name=scope,
num_epochs=10,
debias=True,
sess=sess)
#train and save the model
debiased_model_rf_ad.fit(data_orig_train)
fair_rf_ad = get_fair_metrics_and_plot(filename, data_orig_test, debiased_model_rf_ad, plot=False, model_aif=True)
Out[60]:
In [61]:
fair_rf_ad
Out[61]:
In [62]:
from aif360.algorithms.inprocessing import PrejudiceRemover
debiased_model_pr_rf = PrejudiceRemover()
# Train and save the model
debiased_model_pr_rf.fit(data_orig_train)
fair_rf_pr = get_fair_metrics_and_plot(filename, data_orig_test, debiased_model_pr_rf, plot=False, model_aif=True)
fair_rf_pr
Out[62]:
Out[62]:
#¶
In [63]:
y_pred = debiased_model_pr_rf.predict(data_orig_test)
data_orig_test_pred = data_orig_test.copy(deepcopy=True)
In [64]:
# Prediction with the original RandomForest model
scores = np.zeros_like(data_orig_test.labels)
scores = mdl_rf.predict_proba(data_orig_test.features)[:,1].reshape(-1,1)
data_orig_test_pred.scores = scores
preds = np.zeros_like(data_orig_test.labels)
preds = mdl_rf.predict(data_orig_test.features).reshape(-1,1)
data_orig_test_pred.labels = preds
def format_probs(probs1):
probs1 = np.array(probs1)
probs0 = np.array(1-probs1)
return np.concatenate((probs0, probs1), axis=1)
POST PROCESSING¶
In [65]:
from aif360.algorithms.postprocessing import EqOddsPostprocessing
EOPP_rf = EqOddsPostprocessing(privileged_groups = privileged_groups,
unprivileged_groups = unprivileged_groups,
seed=40)
EOPP_rf = EOPP_rf.fit(data_orig_test, data_orig_test_pred)
data_transf_test_pred_rf_eopp = EOPP_rf.predict(data_orig_test_pred)
fair_rf_eo = fair_metrics(filename, data_orig_test, data_transf_test_pred_rf_eopp, pred_is_dataset=True)
In [66]:
fair_rf_eo
Out[66]:
In [67]:
from aif360.algorithms.postprocessing import CalibratedEqOddsPostprocessing
cost_constraint = "fnr"
CPP_rf = CalibratedEqOddsPostprocessing(privileged_groups = privileged_groups,
unprivileged_groups = unprivileged_groups,
cost_constraint=cost_constraint,
seed=42)
CPP_rf = CPP_rf.fit(data_orig_test, data_orig_test_pred)
data_transf_test_pred_rf_cpp = CPP_rf.predict(data_orig_test_pred)
fair_rf_ceo = fair_metrics(filename, data_orig_test, data_transf_test_pred_rf_cpp, pred_is_dataset=True)
In [68]:
fair_rf_ceo
Out[68]:
In [69]:
from aif360.algorithms.postprocessing import RejectOptionClassification
ROC_rf = RejectOptionClassification(privileged_groups = privileged_groups,
unprivileged_groups = unprivileged_groups)
ROC_rf = ROC_rf.fit(data_orig_test, data_orig_test_pred)
data_transf_test_pred_rf_roc = ROC_rf.predict(data_orig_test_pred)
fair_rf_roc = fair_metrics(filename, data_orig_test, data_transf_test_pred_rf_roc, pred_is_dataset=True)
print('SUCCESS: completed 1 model.')
In [70]:
fair_rf_roc
Out[70]:
2. XGBoost Classifier¶
In [71]:
from xgboost import XGBClassifier
estimator = XGBClassifier(seed=40)
parameters = {
'max_depth': range (2, 10, 2),
'n_estimators': range(60, 240, 40),
'learning_rate': [0.1, 0.01, 0.05]
}
grid_search = GridSearchCV(
estimator=estimator,
param_grid=parameters,
scoring = 'recall',
cv = 5,
verbose=0
)
model_xg=grid_search
In [72]:
mdl_xgb = model_xg.fit(data_orig_train.features, data_orig_train.labels.ravel())
In [73]:
conf_mat_xg = confusion_matrix(data_orig_test.labels.ravel(), model_xg.predict(data_orig_test.features))
conf_mat_xg
from sklearn.metrics import accuracy_score
print(accuracy_score(data_orig_test.labels.ravel(), model_xg.predict(data_orig_test.features)))
Out[73]:
2.a. Feature importance of model¶
In [74]:
importances_xg = model_xg.best_estimator_.feature_importances_
indices_xg = np.argsort(importances_xg)
features = data_orig_train.feature_names
#https://stackoverflow.com/questions/48377296/get-feature-importance-from-gridsearchcv
In [75]:
importances_xg
Out[75]:
In [76]:
importances_xg[indices_xg]
Out[76]:
In [77]:
features
Out[77]:
In [78]:
plt.figure(figsize=(20,30))
plt.title('Feature Importances')
plt.barh(range(len(indices_xg)), importances_xg[indices_xg], color='b', align='center')
plt.yticks(range(len(indices_xg)), [features[i] for i in indices_xg])
plt.xlabel('Relative Importance')
plt.show()
Out[78]:
Out[78]:
Out[78]:
Out[78]:
Out[78]:
2.b. Model Explainability/interpretability¶
2.b.1 Using SHAP (SHapley Additive exPlanations)¶
In [79]:
import shap
xg_shap_values_t1 = shap.KernelExplainer(mdl_xgb.predict,data_orig_train.features)
Test data interpretation¶
In [80]:
xgb_explainer = shap.KernelExplainer(mdl_xgb.predict, data_orig_test.features)
xgb_shap_values = xgb_explainer.shap_values(data_orig_test.features,nsamples=10)
#https://towardsdatascience.com/explain-any-models-with-the-shap-values-use-the-kernelexplainer-79de9464897a
In [81]:
xgb_shap_values
Out[81]:
In [82]:
shap.initjs()
shap.force_plot(xgb_explainer.expected_value,xgb_shap_values[0,:], data_orig_test.features[0],data_orig_test.feature_names,link='logit')
#https://github.com/slundberg/shap
#https://github.com/slundberg/shap/issues/279
Out[82]:
In [83]:
shap.initjs()
shap.force_plot(xgb_explainer.expected_value,xgb_shap_values[1,:], data_orig_test.features[1],data_orig_test.feature_names,link='logit')
Out[83]:
In [84]:
shap.force_plot(xgb_explainer.expected_value,
xgb_shap_values, data_orig_test.features[:,:],feature_names = data_orig_test.feature_names)
Out[84]:
In [85]:
p = shap.summary_plot(xgb_shap_values, data_orig_test.features, feature_names=data_orig_test.feature_names,plot_type="bar")
display(p)
The variables with higher impact are the ones in the top.
In [86]:
shap.plots._waterfall.waterfall_legacy(xgb_explainer.expected_value, xgb_shap_values[0,:],feature_names=data_orig_test.feature_names)
Here credit history none/paid is moving target outcome towards right i.e., 1.
Interpretation of graph: https://shap.readthedocs.io/en/latest/example_notebooks/overviews/An%20introduction%20to%20explainable%20AI%20with%20Shapley%20values.html
f(x)- model output impacted by features; E(f(x))- expected output.
One the fundemental properties of Shapley values is that they always sum up to the difference between the game outcome when all players are present and the game outcome when no players are present. For machine learning models this means that SHAP values of all the input features will always sum up to the difference between baseline (expected) model output and the current model output for the prediction being explained.
In [87]:
shap.plots._waterfall.waterfall_legacy(xgb_explainer.expected_value, xgb_shap_values[1],feature_names=data_orig_test.feature_names)
Here Credit History and Age are moving the target result towards right.
2.b.2 Using ELI5¶
In [88]:
#!pip install eli5
import eli5
from eli5.sklearn import PermutationImportance
In [89]:
perm_xgb = PermutationImportance(mdl_xgb).fit(data_orig_test.features, data_orig_test.labels.ravel())
Feature Importance¶
In [90]:
perm_imp_2=eli5.show_weights(perm_xgb,feature_names = data_orig_test.feature_names)
perm_imp_2
plt.show()
Out[90]:
2.c. Measuring fairness ¶
Of Baseline model ¶
In [91]:
import pandas as pd
import csv
import os
import numpy as np
import sys
from aif360.metrics import *
from sklearn.metrics import confusion_matrix, accuracy_score, f1_score, roc_curve, auc
plot_model_performance(mdl_xgb, X_test, y_test)
In [92]:
fair_xg = get_fair_metrics_and_plot(filename, data_orig_test, model_xg)
fair_xg
Out[92]:
PRE PROCESSING¶
In [93]:
### Reweighing
from aif360.algorithms.preprocessing import Reweighing
RW_xg = Reweighing(unprivileged_groups=unprivileged_groups, privileged_groups=privileged_groups)
data_transf_train_xg_rw = RW_xg.fit_transform(data_orig_train)
#train and save model
xg_transf_rw = model_xg.fit(data_transf_train_xg_rw.features,
data_transf_train_xg_rw.labels.ravel())
data_transf_test_xg_rw = RW_xg.transform(data_orig_test)
fair_xg_rw = get_fair_metrics_and_plot(filename, data_transf_test_xg_rw, xg_transf_rw, plot=False)
In [94]:
fair_xg_rw
Out[94]:
In [95]:
from aif360.algorithms.preprocessing import DisparateImpactRemover
DIR_xg = DisparateImpactRemover()
data_transf_train_xg_dir = DIR_xg.fit_transform(data_orig_train)
# Train and save the model
xg_transf_dir = model_xg.fit(data_transf_train_xg_dir.features,data_transf_train_xg_dir.labels.ravel())
In [96]:
fair_dir_xg_dir = get_fair_metrics_and_plot(filename,data_orig_test, xg_transf_dir, plot=False)
fair_dir_xg_dir
Out[96]:
INPROCESSING¶
In [97]:
#!pip install --user --upgrade tensorflow==1.15.0
#2.2.0
#!pip uninstall tensorflow
In [98]:
#!pip install "tensorflow==1.15"
#!pip install --upgrade tensorflow-hub
In [99]:
#%tensorflow_version 1.15
import tensorflow as tf
#from tensorflow.compat.v1 import variable_scope
print('Using TensorFlow version', tf.__version__)
In [100]:
#sess = tf.compat.v1.Session()
#import tensorflow as tf
sess = tf.compat.v1.Session()
In [101]:
#import tensorflow as tf
#sess = tf.Session()
tf.compat.v1.reset_default_graph()
In [102]:
from aif360.algorithms.inprocessing.adversarial_debiasing import AdversarialDebiasing
#with tf.variable_scope('debiased_classifier',reuse=tf.AUTO_REUSE):
with tf.compat.v1.Session() as sess:
with tf.variable_scope('scope1',reuse=tf.AUTO_REUSE) as scope:
debiased_model_xg_ad = AdversarialDebiasing(privileged_groups = privileged_groups,
unprivileged_groups = unprivileged_groups,
scope_name=scope,
num_epochs=10,
debias=True,
sess=sess)
#train and save the model
debiased_model_xg_ad.fit(data_orig_train)
fair_xg_ad = get_fair_metrics_and_plot(filename, data_orig_test, debiased_model_xg_ad, plot=False, model_aif=True)
Out[102]:
In [103]:
fair_xg_ad
Out[103]:
In [104]:
from aif360.algorithms.inprocessing import PrejudiceRemover
debiased_model_pr_xg = PrejudiceRemover()
# Train and save the model
debiased_model_pr_xg.fit(data_orig_train)
fair_xg_pr = get_fair_metrics_and_plot(filename, data_orig_test, debiased_model_pr_xg, plot=False, model_aif=True)
fair_xg_pr
Out[104]:
Out[104]:
#¶
In [105]:
y_pred = debiased_model_pr_xg.predict(data_orig_test)
data_orig_test_pred = data_orig_test.copy(deepcopy=True)
In [106]:
# Prediction with the original RandomForest model
scores = np.zeros_like(data_orig_test.labels)
scores = mdl_xgb.predict_proba(data_orig_test.features)[:,1].reshape(-1,1)
data_orig_test_pred.scores = scores
preds = np.zeros_like(data_orig_test.labels)
preds = mdl_xgb.predict(data_orig_test.features).reshape(-1,1)
data_orig_test_pred.labels = preds
def format_probs(probs1):
probs1 = np.array(probs1)
probs0 = np.array(1-probs1)
return np.concatenate((probs0, probs1), axis=1)
POST PROCESSING¶
In [107]:
from aif360.algorithms.postprocessing import EqOddsPostprocessing
EOPP_xg = EqOddsPostprocessing(privileged_groups = privileged_groups,
unprivileged_groups = unprivileged_groups,
seed=40)
EOPP_xg = EOPP_xg.fit(data_orig_test, data_orig_test_pred)
data_transf_test_pred_xg_eopp = EOPP_xg.predict(data_orig_test_pred)
fair_xg_eo = fair_metrics(filename, data_orig_test, data_transf_test_pred_xg_eopp, pred_is_dataset=True)
In [108]:
fair_xg_eo
Out[108]:
In [109]:
from aif360.algorithms.postprocessing import CalibratedEqOddsPostprocessing
cost_constraint = "fnr"
CPP_xg = CalibratedEqOddsPostprocessing(privileged_groups = privileged_groups,
unprivileged_groups = unprivileged_groups,
cost_constraint=cost_constraint,
seed=42)
CPP_xg = CPP_xg.fit(data_orig_test, data_orig_test_pred)
data_transf_test_pred_xg_cpp = CPP_xg.predict(data_orig_test_pred)
fair_xg_ceo = fair_metrics(filename, data_orig_test, data_transf_test_pred_xg_cpp, pred_is_dataset=True)
In [110]:
fair_xg_ceo
Out[110]:
In [111]:
from aif360.algorithms.postprocessing import RejectOptionClassification
ROC_xg = RejectOptionClassification(privileged_groups = privileged_groups,
unprivileged_groups = unprivileged_groups)
ROC_xg = ROC_xg.fit(data_orig_test, data_orig_test_pred)
data_transf_test_pred_xg_roc = ROC_xg.predict(data_orig_test_pred)
fair_xg_roc = fair_metrics(filename, data_orig_test, data_transf_test_pred_xg_roc, pred_is_dataset=True)
print('SUCCESS: completed 1 model.')
In [112]:
fair_xg_roc
Out[112]:
In [ ]:
In [ ]:
3. XGBOOST with out hyper-parameter tuning¶
In [113]:
from xgboost import XGBClassifier
model_xgb2 = XGBClassifier(seed=40)
In [114]:
mdl_xgb2 = model_xgb2.fit(data_orig_train.features, data_orig_train.labels.ravel())
In [115]:
conf_mat_xg2 = confusion_matrix(data_orig_test.labels.ravel(), model_xgb2.predict(data_orig_test.features))
conf_mat_xg2
from sklearn.metrics import accuracy_score
print(accuracy_score(data_orig_test.labels.ravel(), model_xgb2.predict(data_orig_test.features)))
Out[115]:
3.a. Feature importance of model¶
In [116]:
importances_xg2 = model_xgb2.feature_importances_
indices_xg2 = np.argsort(importances_xg2)
features2 = data_orig_train.feature_names
#https://stackoverflow.com/questions/48377296/get-feature-importance-from-gridsearchcv
In [117]:
importances_xg2
Out[117]:
In [118]:
importances_xg2[indices_xg2]
Out[118]:
In [119]:
features2
Out[119]:
In [120]:
plt.figure(figsize=(20,30))
plt.title('Feature Importances')
plt.barh(range(len(indices_xg2)), importances_xg2[indices_xg2], color='b', align='center')
plt.yticks(range(len(indices_xg2)), [features2[i] for i in indices_xg2])
plt.xlabel('Relative Importance')
plt.show()
Out[120]:
Out[120]:
Out[120]:
Out[120]:
Out[120]:
3.b. Model Explainability/interpretability¶
3.b.1 Using SHAP (SHapley Additive exPlanations)¶
In [121]:
import shap
xg_shap_values_t = shap.KernelExplainer(mdl_xgb2.predict,data_orig_train.features)
Test data interpretation¶
In [122]:
xgb_explainer2 = shap.KernelExplainer(mdl_xgb2.predict, data_orig_test.features)
xgb_shap_values2 = xgb_explainer2.shap_values(data_orig_test.features,nsamples=10)
#https://towardsdatascience.com/explain-any-models-with-the-shap-values-use-the-kernelexplainer-79de9464897a
In [123]:
xgb_shap_values2
Out[123]:
In [124]:
shap.initjs()
shap.force_plot(xgb_explainer2.expected_value,xgb_shap_values2[0,:], data_orig_test.features[0],data_orig_test.feature_names,link='logit')
#https://github.com/slundberg/shap
#https://github.com/slundberg/shap/issues/279
Out[124]:
In [125]:
shap.initjs()
shap.force_plot(xgb_explainer2.expected_value,xgb_shap_values2[1,:], data_orig_test.features[1],data_orig_test.feature_names,link='logit')
Out[125]:
In [126]:
data_orig_test.feature_names
Out[126]:
In [127]:
shap.force_plot(xgb_explainer2.expected_value,
xgb_shap_values2, data_orig_test.features[:,:],feature_names = data_orig_test.feature_names)
Out[127]:
In [128]:
p = shap.summary_plot(xgb_shap_values2, data_orig_test.features, feature_names=data_orig_test.feature_names,plot_type="bar")
display(p)
The variables with higher impact are at the top.
In [129]:
shap.plots._waterfall.waterfall_legacy(xgb_explainer2.expected_value, xgb_shap_values2[0,:],feature_names=data_orig_test.feature_names)
Interpretation of graph: https://shap.readthedocs.io/en/latest/example_notebooks/overviews/An%20introduction%20to%20explainable%20AI%20with%20Shapley%20values.html
f(x)- model output impacted by features; E(f(x))- expected output.
One the fundemental properties of Shapley values is that they always sum up to the difference between the game outcome when all players are present and the game outcome when no players are present. For machine learning models this means that SHAP values of all the input features will always sum up to the difference between baseline (expected) model output and the current model output for the prediction being explained.
In [130]:
shap.plots._waterfall.waterfall_legacy(xgb_explainer2.expected_value, xgb_shap_values2[1],feature_names=data_orig_test.feature_names)
3.b.2 Using ELI5¶
In [131]:
#!pip install eli5
import eli5
from eli5.sklearn import PermutationImportance
In [132]:
perm_xgb2 = PermutationImportance(mdl_xgb2).fit(data_orig_test.features, data_orig_test.labels.ravel())
Feature Importance¶
In [133]:
perm_imp_3=eli5.show_weights(perm_xgb2,feature_names = data_orig_test.feature_names)
perm_imp_3
plt.show()
Out[133]:
Explaining individual predictions¶
In [134]:
from eli5 import show_prediction
show_prediction(mdl_xgb2, data_orig_test.features[1], show_feature_values=True,feature_names = data_orig_test.feature_names)
Out[134]:
In [ ]:
3.c. Measuring fairness ¶
Of Baseline model ¶
In [135]:
import pandas as pd
import csv
import os
import numpy as np
import sys
from aif360.metrics import *
from sklearn.metrics import confusion_matrix, accuracy_score, f1_score, roc_curve, auc
plot_model_performance(mdl_xgb2, X_test, y_test)
In [136]:
fair_xg2 = get_fair_metrics_and_plot(filename, data_orig_test, mdl_xgb2)
fair_xg2
Out[136]:
PRE PROCESSING¶
In [137]:
### Reweighing
from aif360.algorithms.preprocessing import Reweighing
RW_xg2 = Reweighing(unprivileged_groups=unprivileged_groups, privileged_groups=privileged_groups)
data_transf_train_xg2_rw = RW_xg2.fit_transform(data_orig_train)
#train and save model
xg2_transf_rw = model_xgb2.fit(data_transf_train_xg2_rw.features,
data_transf_train_xg2_rw.labels.ravel())
data_transf_test_xg2_rw = RW_xg2.transform(data_orig_test)
fair_xg2_rw = get_fair_metrics_and_plot(filename, data_transf_test_xg2_rw, xg2_transf_rw, plot=False)
In [138]:
fair_xg2_rw
Out[138]:
In [139]:
from aif360.algorithms.preprocessing import DisparateImpactRemover
DIR_xg2 = DisparateImpactRemover()
data_transf_train_xg2_dir = DIR_xg2.fit_transform(data_orig_train)
# Train and save the model
xg2_transf_dir = model_xgb2.fit(data_transf_train_xg2_dir.features,data_transf_train_xg2_dir.labels.ravel())
In [140]:
fair_dir_xg2_dir = get_fair_metrics_and_plot(filename,data_orig_test, xg2_transf_dir, plot=False)
fair_dir_xg2_dir
Out[140]:
INPROCESSING¶
In [141]:
#!pip install --user --upgrade tensorflow==1.15.0
#2.2.0
#!pip uninstall tensorflow
In [142]:
#!pip install "tensorflow==1.15"
#!pip install --upgrade tensorflow-hub
In [143]:
#%tensorflow_version 1.15
import tensorflow as tf
#from tensorflow.compat.v1 import variable_scope
print('Using TensorFlow version', tf.__version__)
In [144]:
#sess = tf.compat.v1.Session()
#import tensorflow as tf
sess = tf.compat.v1.Session()
In [145]:
#import tensorflow as tf
#sess = tf.Session()
tf.compat.v1.reset_default_graph()
In [146]:
from aif360.algorithms.inprocessing.adversarial_debiasing import AdversarialDebiasing
#with tf.variable_scope('debiased_classifier',reuse=tf.AUTO_REUSE):
with tf.compat.v1.Session() as sess:
with tf.variable_scope('scope1',reuse=tf.AUTO_REUSE) as scope:
debiased_model_xg2_ad = AdversarialDebiasing(privileged_groups = privileged_groups,
unprivileged_groups = unprivileged_groups,
scope_name=scope,
num_epochs=10,
debias=True,
sess=sess)
#train and save the model
debiased_model_xg2_ad.fit(data_orig_train)
fair_xg2_ad = get_fair_metrics_and_plot(filename, data_orig_test, debiased_model_xg2_ad, plot=False, model_aif=True)
Out[146]:
In [147]:
fair_xg2_ad
Out[147]:
In [148]:
from aif360.algorithms.inprocessing import PrejudiceRemover
debiased_model_pr_xg2 = PrejudiceRemover()
# Train and save the model
debiased_model_pr_xg2.fit(data_orig_train)
fair_xg2_pr = get_fair_metrics_and_plot(filename, data_orig_test, debiased_model_pr_xg2, plot=False, model_aif=True)
fair_xg2_pr
Out[148]:
Out[148]:
#¶
In [149]:
y_pred = debiased_model_pr_xg2.predict(data_orig_test)
data_orig_test_pred = data_orig_test.copy(deepcopy=True)
In [150]:
# Prediction with the original RandomForest model
scores = np.zeros_like(data_orig_test.labels)
scores = mdl_xgb2.predict_proba(data_orig_test.features)[:,1].reshape(-1,1)
data_orig_test_pred.scores = scores
preds = np.zeros_like(data_orig_test.labels)
preds = mdl_xgb2.predict(data_orig_test.features).reshape(-1,1)
data_orig_test_pred.labels = preds
def format_probs(probs1):
probs1 = np.array(probs1)
probs0 = np.array(1-probs1)
return np.concatenate((probs0, probs1), axis=1)
POST PROCESSING¶
In [151]:
from aif360.algorithms.postprocessing import EqOddsPostprocessing
EOPP_xg2 = EqOddsPostprocessing(privileged_groups = privileged_groups,
unprivileged_groups = unprivileged_groups,
seed=40)
EOPP_xg2 = EOPP_xg2.fit(data_orig_test, data_orig_test_pred)
data_transf_test_pred_xg2_eopp = EOPP_xg2.predict(data_orig_test_pred)
fair_xg2_eo = fair_metrics(filename, data_orig_test, data_transf_test_pred_xg2_eopp, pred_is_dataset=True)
In [152]:
fair_xg2_eo
Out[152]:
In [153]:
from aif360.algorithms.postprocessing import CalibratedEqOddsPostprocessing
cost_constraint = "fnr"
CPP_xg2 = CalibratedEqOddsPostprocessing(privileged_groups = privileged_groups,
unprivileged_groups = unprivileged_groups,
cost_constraint=cost_constraint,
seed=42)
CPP_xg2 = CPP_xg2.fit(data_orig_test, data_orig_test_pred)
data_transf_test_pred_xg2_cpp = CPP_xg2.predict(data_orig_test_pred)
fair_xg2_ceo = fair_metrics(filename, data_orig_test, data_transf_test_pred_xg2_cpp, pred_is_dataset=True)
In [154]:
fair_xg2_ceo
Out[154]:
In [155]:
from aif360.algorithms.postprocessing import RejectOptionClassification
ROC_xg2 = RejectOptionClassification(privileged_groups = privileged_groups,
unprivileged_groups = unprivileged_groups)
ROC_xg2 = ROC_xg2.fit(data_orig_test, data_orig_test_pred)
data_transf_test_pred_xg2_roc = ROC_xg2.predict(data_orig_test_pred)
fair_xg2_roc = fair_metrics(filename, data_orig_test, data_transf_test_pred_xg2_roc, pred_is_dataset=True)
print('SUCCESS: completed 1 model.')
In [156]:
fair_xg2_roc
Out[156]:
In [ ]:
4. RANDOM FOREST CLASSIFIER MODEL WITH OUT HYPER-PARAMETER TUNING¶
In [157]:
#Creating the classifier
rf_model2 = RandomForestClassifier(random_state=40)
model_rf2=rf_model2
In [158]:
mdl_rf2 = model_rf2.fit(data_orig_train.features, data_orig_train.labels.ravel())
In [159]:
from sklearn.metrics import confusion_matrix
conf_mat_rf2 = confusion_matrix(data_orig_test.labels.ravel(), model_rf2.predict(data_orig_test.features))
conf_mat_rf2
from sklearn.metrics import accuracy_score
print(accuracy_score(data_orig_test.labels.ravel(), model_rf2.predict(data_orig_test.features)))
Out[159]:
In [160]:
unique, counts = np.unique(data_orig_test.labels.ravel(), return_counts=True)
dict(zip(unique, counts))
Out[160]:
4.a. Model Explainability/interpretability¶
4.a.1 Using SHAP (SHapley Additive exPlanations)¶
In [161]:
import shap
rf_shap_values_t2 = shap.KernelExplainer(mdl_rf2.predict,data_orig_train.features)
Test data interpretation¶
In [162]:
rf_explainer2 = shap.KernelExplainer(mdl_rf2.predict, data_orig_test.features)
rf_shap_values2 = rf_explainer2.shap_values(data_orig_test.features,nsamples=10)
#https://towardsdatascience.com/explain-any-models-with-the-shap-values-use-the-kernelexplainer-79de9464897a
In [163]:
rf_shap_values2
Out[163]:
In [164]:
rf_explainer2.expected_value
rf_shap_values2
Out[164]:
Out[164]:
In [165]:
shap.initjs()
shap.force_plot(rf_explainer2.expected_value,rf_shap_values2[0,:], data_orig_test.features[0],data_orig_test.feature_names,link='logit')
#https://github.com/slundberg/shap
#https://github.com/slundberg/shap/issues/279
Out[165]:
In [166]:
shap.initjs()
shap.force_plot(rf_explainer2.expected_value,rf_shap_values2[1,:], data_orig_test.features[1],data_orig_test.feature_names,link='logit')
Out[166]:
In [167]:
shap.initjs()
shap.force_plot(rf_explainer2.expected_value,rf_shap_values2[2,:], data_orig_test.features[2],data_orig_test.feature_names,link='logit')
Out[167]:
In [168]:
data_orig_test.feature_names
Out[168]:
In [169]:
shap.force_plot(rf_explainer2.expected_value,
rf_shap_values2, data_orig_test.features[:,:],feature_names = data_orig_test.feature_names)
Out[169]:
In [170]:
p = shap.summary_plot(rf_shap_values2, data_orig_test.features, feature_names=data_orig_test.feature_names,plot_type="bar")
display(p)
Variables with higher impact are displayed at the top.
In [171]:
shap.plots._waterfall.waterfall_legacy(rf_explainer2.expected_value, rf_shap_values2[0,:],feature_names=data_orig_test.feature_names)
Interpretation of graph: https://shap.readthedocs.io/en/latest/example_notebooks/overviews/An%20introduction%20to%20explainable%20AI%20with%20Shapley%20values.html
f(x)- model output impacted by features; E(f(x))- expected output.
One the fundemental properties of Shapley values is that they always sum up to the difference between the game outcome when all players are present and the game outcome when no players are present. For machine learning models this means that SHAP values of all the input features will always sum up to the difference between baseline (expected) model output and the current model output for the prediction being explained.
In [172]:
shap.plots._waterfall.waterfall_legacy(rf_explainer2.expected_value, rf_shap_values2[1],feature_names=data_orig_test.feature_names)
4.a.2 Using ELI5¶
In [173]:
#!pip install eli5
import eli5
from eli5.sklearn import PermutationImportance
In [174]:
perm_rf2 = PermutationImportance(mdl_rf2).fit(data_orig_test.features, data_orig_test.labels.ravel())
In [175]:
data_orig_test.labels[:10,:].ravel()
Out[175]:
Feature Importance¶
In [176]:
perm_imp_11=eli5.show_weights(perm_rf2,feature_names = data_orig_test.feature_names)
perm_imp_11
plt.show()
Out[176]:
Explaining individual predictions¶
In [177]:
show_prediction(mdl_rf2, data_orig_test.features[0], show_feature_values=True,feature_names = data_orig_test.feature_names)
Out[177]:
In [178]:
from eli5 import show_prediction
show_prediction(mdl_rf2, data_orig_test.features[1], show_feature_values=True,feature_names = data_orig_test.feature_names)
Out[178]:
4.b. Measuring fairness ¶
Of Baseline model ¶
In [179]:
import pandas as pd
import csv
import os
import numpy as np
import sys
from aif360.metrics import *
from sklearn.metrics import confusion_matrix, accuracy_score, f1_score, roc_curve, auc
plot_model_performance(mdl_rf2, X_test, y_test)
In [180]:
fair = get_fair_metrics_and_plot(filename, data_orig_test, mdl_rf2)
fair
Out[180]:
In [181]:
type(data_orig_train)
Out[181]:
PRE PROCESSING¶
In [182]:
### Reweighing
from aif360.algorithms.preprocessing import Reweighing
RW_rf2 = Reweighing(unprivileged_groups=unprivileged_groups, privileged_groups=privileged_groups)
data_transf_train_rf2_rw = RW_rf2.fit_transform(data_orig_train)
#train and save model
rf2_transf_rw = model_rf2.fit(data_transf_train_rf2_rw.features,
data_transf_train_rf2_rw.labels.ravel())
data_transf_test_rf2_rw = RW_rf2.transform(data_orig_test)
fair_rf2_rw = get_fair_metrics_and_plot(filename, data_transf_test_rf2_rw, rf2_transf_rw, plot=False)
In [183]:
fair_rf_rw
Out[183]:
In [184]:
from aif360.algorithms.preprocessing import DisparateImpactRemover
DIR_rf2 = DisparateImpactRemover()
data_transf_train_rf2_dir = DIR_rf2.fit_transform(data_orig_train)
# Train and save the model
rf2_transf_dir = model_rf2.fit(data_transf_train_rf2_dir.features,data_transf_train_rf2_dir.labels.ravel())
In [185]:
fair_dir_rf2_dir = get_fair_metrics_and_plot(filename,data_orig_test, rf2_transf_dir, plot=False)
fair_dir_rf2_dir
Out[185]:
In [186]:
conf_mat_rf2_dir = confusion_matrix(data_orig_test.labels.ravel(), rf2_transf_dir.predict(data_orig_test.features))
conf_mat_rf2_dir
from sklearn.metrics import accuracy_score
print(accuracy_score(data_orig_test.labels.ravel(), rf2_transf_dir.predict(data_orig_test.features)))
Out[186]:
INPROCESSING¶
In [187]:
#!pip install --user --upgrade tensorflow==1.15.0
#2.2.0
#!pip uninstall tensorflow
In [188]:
#!pip install "tensorflow==1.15"
#!pip install --upgrade tensorflow-hub
In [189]:
#%tensorflow_version 1.15
import tensorflow as tf
#from tensorflow.compat.v1 import variable_scope
print('Using TensorFlow version', tf.__version__)
In [190]:
#sess = tf.compat.v1.Session()
#import tensorflow as tf
sess = tf.compat.v1.Session()
In [191]:
#import tensorflow as tf
#sess = tf.Session()
tf.compat.v1.reset_default_graph()
In [192]:
from aif360.algorithms.inprocessing.adversarial_debiasing import AdversarialDebiasing
#with tf.variable_scope('debiased_classifier',reuse=tf.AUTO_REUSE):
with tf.compat.v1.Session() as sess:
with tf.variable_scope('scope1',reuse=tf.AUTO_REUSE) as scope:
debiased_model_rf2_ad = AdversarialDebiasing(privileged_groups = privileged_groups,
unprivileged_groups = unprivileged_groups,
scope_name=scope,
num_epochs=10,
debias=True,
sess=sess)
#train and save the model
debiased_model_rf2_ad.fit(data_orig_train)
fair_rf2_ad = get_fair_metrics_and_plot(filename, data_orig_test, debiased_model_rf2_ad, plot=False, model_aif=True)
Out[192]:
In [193]:
fair_rf2_ad
Out[193]:
In [194]:
from aif360.algorithms.inprocessing import PrejudiceRemover
debiased_model_pr_rf2 = PrejudiceRemover()
# Train and save the model
debiased_model_pr_rf2.fit(data_orig_train)
fair_rf2_pr = get_fair_metrics_and_plot(filename, data_orig_test, debiased_model_pr_rf2, plot=False, model_aif=True)
fair_rf2_pr
Out[194]:
Out[194]:
#¶
In [195]:
y_pred = debiased_model_pr_rf2.predict(data_orig_test)
data_orig_test_pred = data_orig_test.copy(deepcopy=True)
In [196]:
# Prediction with the original RandomForest model
scores = np.zeros_like(data_orig_test.labels)
scores = mdl_rf2.predict_proba(data_orig_test.features)[:,1].reshape(-1,1)
data_orig_test_pred.scores = scores
preds = np.zeros_like(data_orig_test.labels)
preds = mdl_rf2.predict(data_orig_test.features).reshape(-1,1)
data_orig_test_pred.labels = preds
def format_probs(probs1):
probs1 = np.array(probs1)
probs0 = np.array(1-probs1)
return np.concatenate((probs0, probs1), axis=1)
POST PROCESSING¶
In [197]:
from aif360.algorithms.postprocessing import EqOddsPostprocessing
EOPP_rf2 = EqOddsPostprocessing(privileged_groups = privileged_groups,
unprivileged_groups = unprivileged_groups,
seed=40)
EOPP_rf2 = EOPP_rf2.fit(data_orig_test, data_orig_test_pred)
data_transf_test_pred_rf2_eopp = EOPP_rf2.predict(data_orig_test_pred)
fair_rf2_eo = fair_metrics(filename, data_orig_test, data_transf_test_pred_rf2_eopp, pred_is_dataset=True)
In [198]:
fair_rf2_eo
Out[198]:
In [199]:
from aif360.algorithms.postprocessing import CalibratedEqOddsPostprocessing
cost_constraint = "fnr"
CPP_rf2 = CalibratedEqOddsPostprocessing(privileged_groups = privileged_groups,
unprivileged_groups = unprivileged_groups,
cost_constraint=cost_constraint,
seed=42)
CPP_rf2 = CPP_rf2.fit(data_orig_test, data_orig_test_pred)
data_transf_test_pred_rf2_cpp = CPP_rf2.predict(data_orig_test_pred)
fair_rf2_ceo = fair_metrics(filename, data_orig_test, data_transf_test_pred_rf2_cpp, pred_is_dataset=True)
In [200]:
fair_rf2_ceo
Out[200]:
In [201]:
from aif360.algorithms.postprocessing import RejectOptionClassification
ROC_rf2 = RejectOptionClassification(privileged_groups = privileged_groups,
unprivileged_groups = unprivileged_groups)
ROC_rf2 = ROC_rf2.fit(data_orig_test, data_orig_test_pred)
data_transf_test_pred_rf2_roc = ROC_rf2.predict(data_orig_test_pred)
fair_rf2_roc = fair_metrics(filename, data_orig_test, data_transf_test_pred_rf2_roc, pred_is_dataset=True)
print('SUCCESS: completed 1 model.')
In [202]:
fair_rf2_roc
Out[202]:
5. KNN¶
In [203]:
from sklearn import neighbors
n_neighbors = 15
knn = neighbors.KNeighborsClassifier(n_neighbors, weights='distance')
In [204]:
knn.fit(data_orig_train.features, data_orig_train.labels.ravel())
Out[204]:
In [205]:
conf_mat_knn = confusion_matrix(data_orig_test.labels.ravel(), knn.predict(data_orig_test.features))
conf_mat_knn
from sklearn.metrics import accuracy_score
print(accuracy_score(data_orig_test.labels.ravel(), knn.predict(data_orig_test.features)))
Out[205]:
5.a. Model Explainability/interpretability¶
5.a.1 Using SHAP (SHapley Additive exPlanations)¶
In [206]:
knn_explainer = shap.KernelExplainer(knn.predict, data_orig_test.features)
knn_shap_values = knn_explainer.shap_values(data_orig_test.features,nsamples=10)
In [207]:
#shap.dependence_plot(0, knn_shap_values, data_orig_test.features)
In [208]:
# plot the SHAP values for the 0th observation
shap.force_plot(knn_explainer.expected_value,knn_shap_values[0,:], data_orig_test.features[0],data_orig_test.feature_names,link='logit')
Out[208]:
In [209]:
# plot the SHAP values for the 1st observation
shap.force_plot(knn_explainer.expected_value,knn_shap_values[1,:], data_orig_test.features[1],data_orig_test.feature_names,link='logit')
Out[209]:
In [210]:
shap.force_plot(knn_explainer.expected_value, knn_shap_values, data_orig_test.feature_names,link='logit')
Out[210]:
In [211]:
shap.summary_plot(knn_shap_values, data_orig_test.features,feature_names=data_orig_test.feature_names, plot_type="violin")
Feature Importance
perm_imp_11=eli5.show_weights(knn,feature_names = data_orig_test.feature_names) perm_imp_11 plt.show()
Explaining individual predictions¶
In [212]:
from eli5 import show_prediction
show_prediction(knn, data_orig_test.features[1], show_feature_values=True,feature_names = data_orig_test.feature_names)
Out[212]:
5.b. Measuring fairness ¶
Of Baseline model ¶
In [213]:
import pandas as pd
import csv
import os
import numpy as np
import sys
from aif360.metrics import *
from sklearn.metrics import confusion_matrix, accuracy_score, f1_score, roc_curve, auc
plot_model_performance(knn, X_test, y_test)
In [214]:
fair = get_fair_metrics_and_plot(filename, data_orig_test, knn)
fair
Out[214]:
PRE PROCESSING¶
In [215]:
### Reweighing
from aif360.algorithms.preprocessing import Reweighing
RW_knn = Reweighing(unprivileged_groups=unprivileged_groups, privileged_groups=privileged_groups)
data_transf_train_knn = RW_knn.fit_transform(data_orig_train)
# Train and save the model
knn_transf_rw = knn.fit(data_transf_train_knn.features,
data_transf_train_knn.labels.ravel())
data_transf_test_knn_rw = RW_knn.transform(data_orig_test)
fair_knn_rw = get_fair_metrics_and_plot(filename, data_transf_test_knn_rw, knn_transf_rw, plot=False)
In [216]:
fair_knn_rw
Out[216]:
In [217]:
from aif360.algorithms.preprocessing import DisparateImpactRemover
DIR = DisparateImpactRemover()
data_transf_train_knn_dir = DIR.fit_transform(data_orig_train)
# Train and save the model
knn_transf_dir = knn.fit(data_transf_train_knn_dir.features,
data_transf_train_knn_dir.labels.ravel())
In [218]:
fair_knn_dir = get_fair_metrics_and_plot(filename, data_orig_test, knn_transf_dir, plot=False)
fair_knn_dir
Out[218]:
INPROCESSING¶
In [219]:
#!pip install tensorflow
import tensorflow as tf
#from tensorflow.compat.v1 import variable_scope
print('Using TensorFlow version', tf.__version__)
In [220]:
#sess = tf.compat.v1.Session()
#import tensorflow as tf
sess = tf.compat.v1.Session()
In [221]:
#import tensorflow as tf
#sess = tf.Session()
tf.compat.v1.reset_default_graph()
In [222]:
from aif360.algorithms.inprocessing.adversarial_debiasing import AdversarialDebiasing
#with tf.variable_scope('debiased_classifier',reuse=tf.AUTO_REUSE):
with tf.compat.v1.Session() as sess:
with tf.variable_scope('scope4',reuse=tf.AUTO_REUSE) as scope:
debiased_model_knn_ad = AdversarialDebiasing(privileged_groups = privileged_groups,
unprivileged_groups = unprivileged_groups,
scope_name=scope,
num_epochs=10,
debias=True,
sess=sess)
debiased_model_knn_ad.fit(data_orig_train)
fair_knn_ad = get_fair_metrics_and_plot(filename, data_orig_test, debiased_model_knn_ad, plot=False, model_aif=True)
Out[222]:
In [223]:
fair_knn_ad
Out[223]:
In [224]:
from aif360.algorithms.inprocessing import PrejudiceRemover
debiased_model_knn_pr = PrejudiceRemover()
# Train and save the model
debiased_model_knn_pr.fit(data_orig_train)
fair_knn_pr = get_fair_metrics_and_plot(filename, data_orig_test, debiased_model_knn_pr, plot=False, model_aif=True)
fair_knn_pr
Out[224]:
Out[224]:
#¶
In [225]:
y_pred = debiased_model_knn_pr.predict(data_orig_test)
data_orig_test_pred = data_orig_test.copy(deepcopy=True)
In [226]:
# Prediction with the original RandomForest model
scores = np.zeros_like(data_orig_test.labels)
scores = knn.predict_proba(data_orig_test.features)[:,1].reshape(-1,1)
data_orig_test_pred.scores = scores
preds = np.zeros_like(data_orig_test.labels)
preds = knn.predict(data_orig_test.features).reshape(-1,1)
data_orig_test_pred.labels = preds
def format_probs(probs1):
probs1 = np.array(probs1)
probs0 = np.array(1-probs1)
return np.concatenate((probs0, probs1), axis=1)
POST PROCESSING¶
In [227]:
from aif360.algorithms.postprocessing import EqOddsPostprocessing
EOPP_knn = EqOddsPostprocessing(privileged_groups = privileged_groups,
unprivileged_groups = unprivileged_groups,
seed=40)
EOPP_knn = EOPP_knn.fit(data_orig_test, data_orig_test_pred)
data_transf_test_pred_knn_eop = EOPP_knn.predict(data_orig_test_pred)
fair_knn_eo = fair_metrics(filename, data_orig_test, data_transf_test_pred_knn_eop, pred_is_dataset=True)
In [228]:
fair_knn_eo
Out[228]:
In [229]:
from aif360.algorithms.postprocessing import CalibratedEqOddsPostprocessing
cost_constraint = "fnr"
CPP_knn = CalibratedEqOddsPostprocessing(privileged_groups = privileged_groups,
unprivileged_groups = unprivileged_groups,
cost_constraint=cost_constraint,
seed=40)
CPP_knn = CPP_knn.fit(data_orig_test, data_orig_test_pred)
data_transf_test_pred_knn_cp = CPP_knn.predict(data_orig_test_pred)
fair_knn_ceo = fair_metrics(filename, data_orig_test, data_transf_test_pred_knn_cp, pred_is_dataset=True)
In [230]:
fair_knn_ceo
Out[230]:
In [231]:
from aif360.algorithms.postprocessing import RejectOptionClassification
ROC_knn = RejectOptionClassification(privileged_groups = privileged_groups,
unprivileged_groups = unprivileged_groups)
ROC_knn = ROC_knn.fit(data_orig_test, data_orig_test_pred)
data_transf_test_pred_knn_roc = ROC_knn.predict(data_orig_test_pred)
fair_knn_roc = fair_metrics(filename, data_orig_test, data_transf_test_pred_knn_roc, pred_is_dataset=True)
print('SUCCESS: completed 1 model.')
In [232]:
fair_knn_roc
Out[232]:
6. Logistic Regression¶
In [233]:
from sklearn.linear_model import LogisticRegression
lr = LogisticRegression()
In [234]:
lr.fit(data_orig_train.features, data_orig_train.labels.ravel())
Out[234]:
In [235]:
conf_mat_lr = confusion_matrix(data_orig_test.labels.ravel(), lr.predict(data_orig_test.features))
conf_mat_lr
from sklearn.metrics import accuracy_score
print(accuracy_score(data_orig_test.labels.ravel(), lr.predict(data_orig_test.features)))
Out[235]:
6.a. Model Explainability/interpretability¶
6.a.1 Using SHAP (SHapley Additive exPlanations)¶
In [236]:
lr_explainer = shap.KernelExplainer(lr.predict, data_orig_test.features)
lr_shap_values = lr_explainer.shap_values(data_orig_test.features,nsamples=10)
In [237]:
# plot the SHAP values for the 0th observation
shap.force_plot(lr_explainer.expected_value,lr_shap_values[0,:], data_orig_test.features[0],data_orig_test.feature_names,link='logit')
Out[237]:
In [238]:
# plot the SHAP values for the 1st observation
shap.force_plot(lr_explainer.expected_value,lr_shap_values[1,:], data_orig_test.features[1],data_orig_test.feature_names,link='logit')
Out[238]:
In [239]:
shap.force_plot(lr_explainer.expected_value, lr_shap_values, data_orig_test.feature_names,link='logit')
Out[239]:
In [240]:
shap.summary_plot(lr_shap_values, data_orig_test.features,feature_names=data_orig_test.feature_names, plot_type="violin")
Feature Importance
perm_imp_11=eli5.show_weights(knn,feature_names = data_orig_test.feature_names) perm_imp_11 plt.show()
Explaining individual predictions¶
In [241]:
from eli5 import show_prediction
show_prediction(lr, data_orig_test.features[1], show_feature_values=True,feature_names = data_orig_test.feature_names)
Out[241]:
6.b. Measuring fairness ¶
Of Baseline model ¶
In [242]:
import pandas as pd
import csv
import os
import numpy as np
import sys
from aif360.metrics import *
from sklearn.metrics import confusion_matrix, accuracy_score, f1_score, roc_curve, auc
plot_model_performance(lr, X_test, y_test)
In [243]:
fair_lr = get_fair_metrics_and_plot(filename, data_orig_test, lr)
fair_lr
Out[243]:
PRE PROCESSING¶
In [244]:
### Reweighing
from aif360.algorithms.preprocessing import Reweighing
RW_lr = Reweighing(unprivileged_groups=unprivileged_groups, privileged_groups=privileged_groups)
data_transf_train_lr = RW_lr.fit_transform(data_orig_train)
# Train and save the model
lr_transf_rw = lr.fit(data_transf_train_knn.features,
data_transf_train_knn.labels.ravel())
data_transf_test_lr_rw = RW_lr.transform(data_orig_test)
fair_lr_rw = get_fair_metrics_and_plot(filename, data_transf_test_lr_rw, lr_transf_rw, plot=False)
In [245]:
fair_lr_rw
Out[245]:
In [246]:
from aif360.algorithms.preprocessing import DisparateImpactRemover
DIR = DisparateImpactRemover()
data_transf_train_lr_dir = DIR.fit_transform(data_orig_train)
# Train and save the model
lr_transf_dir = lr.fit(data_transf_train_lr_dir.features,
data_transf_train_lr_dir.labels.ravel())
In [247]:
fair_lr_dir = get_fair_metrics_and_plot(filename, data_orig_test, lr_transf_dir, plot=False)
fair_lr_dir
Out[247]:
INPROCESSING¶
In [248]:
#!pip install tensorflow
import tensorflow as tf
#from tensorflow.compat.v1 import variable_scope
print('Using TensorFlow version', tf.__version__)
In [249]:
#sess = tf.compat.v1.Session()
#import tensorflow as tf
sess = tf.compat.v1.Session()
In [250]:
#import tensorflow as tf
#sess = tf.Session()
tf.compat.v1.reset_default_graph()
In [251]:
from aif360.algorithms.inprocessing.adversarial_debiasing import AdversarialDebiasing
#with tf.variable_scope('debiased_classifier',reuse=tf.AUTO_REUSE):
with tf.compat.v1.Session() as sess:
with tf.variable_scope('scope5',reuse=tf.AUTO_REUSE) as scope:
debiased_model_lr_ad = AdversarialDebiasing(privileged_groups = privileged_groups,
unprivileged_groups = unprivileged_groups,
scope_name=scope,
num_epochs=10,
debias=True,
sess=sess)
debiased_model_lr_ad.fit(data_orig_train)
fair_lr_ad = get_fair_metrics_and_plot(filename, data_orig_test, debiased_model_lr_ad, plot=False, model_aif=True)
Out[251]:
In [252]:
fair_lr_ad
Out[252]:
In [253]:
from aif360.algorithms.inprocessing import PrejudiceRemover
debiased_model_lr_pr = PrejudiceRemover()
# Train and save the model
debiased_model_lr_pr.fit(data_orig_train)
fair_lr_pr = get_fair_metrics_and_plot(filename, data_orig_test, debiased_model_lr_pr, plot=False, model_aif=True)
fair_lr_pr
Out[253]:
Out[253]:
#¶
In [254]:
y_pred = debiased_model_lr_pr.predict(data_orig_test)
data_orig_test_pred = data_orig_test.copy(deepcopy=True)
In [255]:
# Prediction with the original RandomForest model
scores = np.zeros_like(data_orig_test.labels)
scores = lr.predict_proba(data_orig_test.features)[:,1].reshape(-1,1)
data_orig_test_pred.scores = scores
preds = np.zeros_like(data_orig_test.labels)
preds = lr.predict(data_orig_test.features).reshape(-1,1)
data_orig_test_pred.labels = preds
def format_probs(probs1):
probs1 = np.array(probs1)
probs0 = np.array(1-probs1)
return np.concatenate((probs0, probs1), axis=1)
POST PROCESSING¶
In [256]:
from aif360.algorithms.postprocessing import EqOddsPostprocessing
EOPP_lr = EqOddsPostprocessing(privileged_groups = privileged_groups,
unprivileged_groups = unprivileged_groups,
seed=40)
EOPP_lr = EOPP_lr.fit(data_orig_test, data_orig_test_pred)
data_transf_test_pred_lr_eop = EOPP_lr.predict(data_orig_test_pred)
fair_lr_eo = fair_metrics(filename, data_orig_test, data_transf_test_pred_lr_eop, pred_is_dataset=True)
In [257]:
fair_lr_eo
Out[257]:
In [258]:
from aif360.algorithms.postprocessing import CalibratedEqOddsPostprocessing
cost_constraint = "fnr"
CPP_lr = CalibratedEqOddsPostprocessing(privileged_groups = privileged_groups,
unprivileged_groups = unprivileged_groups,
cost_constraint=cost_constraint,
seed=40)
CPP_lr = CPP_lr.fit(data_orig_test, data_orig_test_pred)
data_transf_test_pred_lr_cp = CPP_lr.predict(data_orig_test_pred)
fair_lr_ceo = fair_metrics(filename, data_orig_test, data_transf_test_pred_lr_cp, pred_is_dataset=True)
In [259]:
fair_lr_ceo
Out[259]:
In [260]:
from aif360.algorithms.postprocessing import RejectOptionClassification
ROC_lr = RejectOptionClassification(privileged_groups = privileged_groups,
unprivileged_groups = unprivileged_groups)
ROC_lr = ROC_lr.fit(data_orig_test, data_orig_test_pred)
data_transf_test_pred_lr_roc = ROC_lr.predict(data_orig_test_pred)
fair_lr_roc = fair_metrics(filename, data_orig_test, data_transf_test_pred_lr_roc, pred_is_dataset=True)
print('SUCCESS: completed 1 model.')
In [261]:
fair_lr_roc
Out[261]:
In [ ]:
7. SVM¶
In [262]:
from sklearn.svm import SVC
#gs = grid_search_cv.best_estimator_
svm = SVC(C=0.85, break_ties=False, cache_size=200, class_weight=None, coef0=0.0,
decision_function_shape='ovr', degree=3, gamma='scale', kernel='linear',
max_iter=-1, random_state=42, shrinking=True, tol=0.001, probability=True,
verbose=False)
svm.fit(data_orig_train.features, data_orig_train.labels.ravel())
Out[262]:
In [263]:
conf_mat_svm = confusion_matrix(data_orig_test.labels.ravel(), svm.predict(data_orig_test.features))
conf_mat_svm
from sklearn.metrics import accuracy_score
print(accuracy_score(data_orig_test.labels.ravel(), svm.predict(data_orig_test.features)))
Out[263]:
7.a. Model Explainability/interpretability¶
7.a.1 Using SHAP (SHapley Additive exPlanations)¶
In [264]:
svm_explainer = shap.KernelExplainer(svm.predict, data_orig_test.features)
svm_shap_values = svm_explainer.shap_values(data_orig_test.features,nsamples=10)
In [265]:
# plot the SHAP values for the 0th observation
shap.force_plot(svm_explainer.expected_value,svm_shap_values[0,:], data_orig_test.features[0],data_orig_test.feature_names,link='logit')
Out[265]:
In [266]:
# plot the SHAP values for the 1st observation
shap.force_plot(svm_explainer.expected_value,svm_shap_values[1,:], data_orig_test.features[1],data_orig_test.feature_names,link='logit')
Out[266]:
In [267]:
shap.force_plot(svm_explainer.expected_value, svm_shap_values, data_orig_test.feature_names,link='logit')
Out[267]:
In [268]:
shap.summary_plot(svm_shap_values, data_orig_test.features,feature_names=data_orig_test.feature_names, plot_type="violin")
Feature Importance
perm_imp_11=eli5.show_weights(knn,feature_names = data_orig_test.feature_names) perm_imp_11 plt.show()
Explaining individual predictions¶
In [269]:
from eli5 import show_prediction
show_prediction(svm, data_orig_test.features[1], show_feature_values=True,feature_names = data_orig_test.feature_names)
Out[269]:
7.b. Measuring fairness ¶
Of Baseline model ¶
In [270]:
import pandas as pd
import csv
import os
import numpy as np
import sys
from aif360.metrics import *
from sklearn.metrics import confusion_matrix, accuracy_score, f1_score, roc_curve, auc
plot_model_performance(svm, X_test, y_test)
In [271]:
fair_svm = get_fair_metrics_and_plot(filename, data_orig_test, svm)
fair_svm
Out[271]:
PRE PROCESSING¶
In [272]:
### Reweighing
from aif360.algorithms.preprocessing import Reweighing
RW_svm = Reweighing(unprivileged_groups=unprivileged_groups, privileged_groups=privileged_groups)
data_transf_train_svm = RW_svm.fit_transform(data_orig_train)
# Train and save the model
svm_transf_rw = svm.fit(data_transf_train_knn.features,
data_transf_train_knn.labels.ravel())
data_transf_test_svm_rw = RW_svm.transform(data_orig_test)
fair_svm_rw = get_fair_metrics_and_plot(filename, data_transf_test_svm_rw, svm_transf_rw, plot=False)
In [273]:
fair_svm_rw
Out[273]:
In [274]:
from aif360.algorithms.preprocessing import DisparateImpactRemover
DIR = DisparateImpactRemover()
data_transf_train_svm_dir = DIR.fit_transform(data_orig_train)
# Train and save the model
svm_transf_dir = svm.fit(data_transf_train_svm_dir.features,
data_transf_train_svm_dir.labels.ravel())
In [275]:
fair_svm_dir = get_fair_metrics_and_plot(filename, data_orig_test, svm_transf_dir, plot=False)
fair_svm_dir
Out[275]:
INPROCESSING¶
In [276]:
#!pip install tensorflow
import tensorflow as tf
#from tensorflow.compat.v1 import variable_scope
print('Using TensorFlow version', tf.__version__)
In [277]:
#sess = tf.compat.v1.Session()
#import tensorflow as tf
sess = tf.compat.v1.Session()
In [278]:
#import tensorflow as tf
#sess = tf.Session()
tf.compat.v1.reset_default_graph()
In [279]:
from aif360.algorithms.inprocessing.adversarial_debiasing import AdversarialDebiasing
#with tf.variable_scope('debiased_classifier',reuse=tf.AUTO_REUSE):
with tf.compat.v1.Session() as sess:
with tf.variable_scope('scope6',reuse=tf.AUTO_REUSE) as scope:
debiased_model_svm_ad = AdversarialDebiasing(privileged_groups = privileged_groups,
unprivileged_groups = unprivileged_groups,
scope_name=scope,
num_epochs=10,
debias=True,
sess=sess)
debiased_model_svm_ad.fit(data_orig_train)
fair_svm_ad = get_fair_metrics_and_plot(filename, data_orig_test, debiased_model_svm_ad, plot=False, model_aif=True)
Out[279]:
In [280]:
fair_svm_ad
Out[280]:
In [281]:
from aif360.algorithms.inprocessing import PrejudiceRemover
debiased_model_svm_pr = PrejudiceRemover()
# Train and save the model
debiased_model_svm_pr.fit(data_orig_train)
fair_svm_pr = get_fair_metrics_and_plot(filename, data_orig_test, debiased_model_svm_pr, plot=False, model_aif=True)
fair_svm_pr
Out[281]:
Out[281]:
#¶
In [282]:
y_pred = debiased_model_svm_pr.predict(data_orig_test)
data_orig_test_pred = data_orig_test.copy(deepcopy=True)
In [283]:
# Prediction with the original RandomForest model
scores = np.zeros_like(data_orig_test.labels)
scores = svm.predict_proba(data_orig_test.features)[:,1].reshape(-1,1)
data_orig_test_pred.scores = scores
preds = np.zeros_like(data_orig_test.labels)
preds = svm.predict(data_orig_test.features).reshape(-1,1)
data_orig_test_pred.labels = preds
def format_probs(probs1):
probs1 = np.array(probs1)
probs0 = np.array(1-probs1)
return np.concatenate((probs0, probs1), axis=1)
POST PROCESSING¶
In [284]:
from aif360.algorithms.postprocessing import EqOddsPostprocessing
EOPP_svm = EqOddsPostprocessing(privileged_groups = privileged_groups,
unprivileged_groups = unprivileged_groups,
seed=40)
EOPP_svm = EOPP_svm.fit(data_orig_test, data_orig_test_pred)
data_transf_test_pred_svm_eop = EOPP_svm.predict(data_orig_test_pred)
fair_svm_eo = fair_metrics(filename, data_orig_test, data_transf_test_pred_svm_eop, pred_is_dataset=True)
In [285]:
fair_svm_eo
Out[285]:
In [286]:
from aif360.algorithms.postprocessing import CalibratedEqOddsPostprocessing
cost_constraint = "fnr"
CPP_svm = CalibratedEqOddsPostprocessing(privileged_groups = privileged_groups,
unprivileged_groups = unprivileged_groups,
cost_constraint=cost_constraint,
seed=40)
CPP_svm = CPP_svm.fit(data_orig_test, data_orig_test_pred)
data_transf_test_pred_svm_cp = CPP_svm.predict(data_orig_test_pred)
fair_svm_ceo = fair_metrics(filename, data_orig_test, data_transf_test_pred_svm_cp, pred_is_dataset=True)
In [287]:
fair_svm_ceo
Out[287]:
In [288]:
from aif360.algorithms.postprocessing import RejectOptionClassification
ROC_svm = RejectOptionClassification(privileged_groups = privileged_groups,
unprivileged_groups = unprivileged_groups)
ROC_svm = ROC_svm.fit(data_orig_test, data_orig_test_pred)
data_transf_test_pred_svm_roc = ROC_svm.predict(data_orig_test_pred)
fair_svm_roc = fair_metrics(filename, data_orig_test, data_transf_test_pred_svm_roc, pred_is_dataset=True)
print('SUCCESS: completed 1 model.')
In [289]:
fair_svm_roc
Out[289]:
- If the measure is less than or greater than 0,bias exists. The negative bias denotes that the prediction is biased towards privileged group and positive bias denotes that prediction is biased towards unprivileged group.
Deploy¶
In [290]:
import pickle
pickle.dump(xg_transf_dir,open('xg_hp_dir_age_debiased.pkl','wb'))
xg_dir_model_age=pickle.load(open('xg_hp_dir_age_debiased.pkl','rb'))
In [291]:
data_orig_test.features[0]
Out[291]:
In [292]:
data_orig_test.labels[0].ravel()
Out[292]:
In [295]:
data_orig_test.features[0]
Out[295]:
In [301]:
type(xg_transf_dir)
Out[301]:
In [305]:
pred=xg_transf_dir.predict(np.array([[ "1" , "0" , "1" , "0" , "0" , "1" , "1" , "0" , "0" , "36" , "0" , "0" , "0" , "0.30802245" , "1", "0" ]]))
In [306]:
pred[0]
Out[306]:
In [ ]: